#!/bin/bash
# {{ ansible_managed }}
# 备份验证脚本 - 用于监控基础设施的备份系统
# 功能特性:
# - 验证备份文件完整性 (tar.gz, gz, bz2, xz, sql)
# - 测试恢复功能
# - 检查备份计划合规性
# - 生成验证报告
# - 支持不同验证类型
# - 校验和验证
# - 备份时效性验证
# - 磁盘空间监控
#
# 使用方法: verify.sh [verify_type] [backup_date]
# verify_type: integrity|restore|schedule|full (默认: integrity)
# backup_date: YYYYMMDD (默认: 最新)
#
# 使用示例:
#   verify.sh integrity           # 验证最新备份的完整性
#   verify.sh restore 20240115    # 测试指定日期的恢复功能
#   verify.sh full                # 完整验证 (所有类型)

# 启用严格错误处理模式
set -euo pipefail

# 配置参数
VERIFY_TYPE="${1:-integrity}"                                    # 验证类型，默认为完整性验证
BACKUP_DATE="${2:-latest}"                                       # 备份日期，默认为最新
BACKUP_BASE_DIR="{{ backup_base_dir }}"                         # 备份基础目录
BACKUP_LOG_DIR="{{ backup_log_dir }}"                           # 备份日志目录
TIMESTAMP=$(date +"%Y%m%d_%H%M%S")                              # 当前时间戳
LOG_FILE="${BACKUP_LOG_DIR}/verify_${VERIFY_TYPE}_${TIMESTAMP}.log"  # 验证日志文件
TEST_RESTORE_DIR="/tmp/backup_test_restore_${TIMESTAMP}"        # 测试恢复目录
VERIFICATION_REPORT="${BACKUP_LOG_DIR}/verification_report_${TIMESTAMP}.txt"  # 验证报告文件

# 统计计数器
TOTAL_FILES=0        # 总文件数
VERIFIED_FILES=0     # 已验证文件数
FAILED_FILES=0       # 验证失败文件数
CORRUPTED_FILES=0    # 损坏文件数

# 工具函数定义
# 日志记录函数 - 同时输出到控制台和日志文件
log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"
}

# 错误退出函数 - 记录错误并退出
error_exit() {
    log "ERROR: $1"
    exit 1
}

# 报告记录函数 - 写入验证报告
report() {
    echo "$1" | tee -a "$VERIFICATION_REPORT"
}

# 查找最新备份目录函数
find_latest_backup() {
    local backup_type="$1"    # 备份类型 (daily/weekly/monthly)
    local component="$2"      # 组件名称 (configs/data/databases/logs)
    
    local backup_path="${BACKUP_BASE_DIR}/${backup_type}/${component}"
    
    # 检查备份路径是否存在
    if [[ ! -d "$backup_path" ]]; then
        log "WARNING: No backups found for $component in $backup_path"
        return 1
    fi
    
    # 查找最新的备份目录（按时间戳排序）
    local latest_backup=$(find "$backup_path" -maxdepth 1 -type d -name "*" | sort -r | head -n 1)
    
    if [[ -z "$latest_backup" ]]; then
        log "WARNING: No backup directories found in $backup_path"
        return 1
    fi
    
    echo "$latest_backup"
}

# 文件完整性验证函数
verify_file_integrity() {
    local file="$1"          # 文件路径
    local file_type="$2"     # 文件类型
    
    ((TOTAL_FILES++))        # 增加总文件计数
    
    log "Verifying $file_type: $(basename "$file")"
    
    # 根据文件类型进行相应的完整性检查
    case "$file_type" in
        "tar.gz")
            # 验证 TAR.GZ 压缩文件的完整性
            if tar -tzf "$file" >/dev/null 2>&1; then
                log "✓ TAR.GZ file integrity verified: $(basename "$file")"
                ((VERIFIED_FILES++))
                return 0
            else
                log "✗ TAR.GZ file corrupted: $(basename "$file")"
                ((CORRUPTED_FILES++))
                ((FAILED_FILES++))
                return 1
            fi
            ;;
        "gz")
            # 验证 GZIP 压缩文件的完整性
            if gzip -t "$file" 2>/dev/null; then
                log "✓ GZIP file integrity verified: $(basename "$file")"
                ((VERIFIED_FILES++))
                return 0
            else
                log "✗ GZIP file corrupted: $(basename "$file")"
                ((CORRUPTED_FILES++))
                ((FAILED_FILES++))
                return 1
            fi
            ;;
        "bz2")
            if bzip2 -t "$file" 2>/dev/null; then
                log "✓ BZIP2 file integrity verified: $(basename "$file")"
                ((VERIFIED_FILES++))
                return 0
            else
                log "✗ BZIP2 file corrupted: $(basename "$file")"
                ((CORRUPTED_FILES++))
                ((FAILED_FILES++))
                return 1
            fi
            ;;
        "xz")
            if xz -t "$file" 2>/dev/null; then
                log "✓ XZ file integrity verified: $(basename "$file")"
                ((VERIFIED_FILES++))
                return 0
            else
                log "✗ XZ file corrupted: $(basename "$file")"
                ((CORRUPTED_FILES++))
                ((FAILED_FILES++))
                return 1
            fi
            ;;
        "sql")
            # SQL 文件基本语法检查
            if grep -q "CREATE\|INSERT\|UPDATE\|DELETE" "$file" 2>/dev/null; then
                log "✓ SQL file appears valid: $(basename "$file")"
                ((VERIFIED_FILES++))
                return 0
            else
                log "✗ SQL file appears invalid: $(basename "$file")"
                ((FAILED_FILES++))
                return 1
            fi
            ;;
        *)
            # 未知文件类型，跳过验证
            log "? Unknown file type, skipping verification: $(basename "$file")"
            return 0
            ;;
    esac
}

# 校验和验证函数
verify_checksum() {
    local file="$1"                           # 要验证的文件
    local checksum_file="${file}.sha256"      # 对应的校验和文件
    
    # 检查是否存在校验和文件
    if [[ -f "$checksum_file" ]]; then
        log "Verifying checksum for $(basename "$file")"
        
        # 使用 sha256sum 验证文件完整性
        if sha256sum -c "$checksum_file" >/dev/null 2>&1; then
            log "✓ Checksum verified: $(basename "$file")"
            return 0
        else
            log "✗ Checksum verification failed: $(basename "$file")"
            ((FAILED_FILES++))
            return 1
        fi
    else
        log "! No checksum file found for $(basename "$file")"
        return 0
    fi
}

# 备份完整性验证函数
verify_backup_completeness() {
    local backup_dir="$1"    # 备份目录
    local component="$2"     # 组件名称
    
    log "Verifying backup completeness for $component in $backup_dir"
    
    # 检查是否存在清单文件
    if [[ -f "${backup_dir}/MANIFEST.txt" ]]; then
        log "✓ Backup manifest found"
        
        # 验证清单中列出的所有文件是否存在
        while IFS= read -r line; do
            if [[ "$line" =~ ^/.* ]]; then  # 文件路径行
                local file_path=$(echo "$line" | awk '{print $NF}')
                if [[ -f "$file_path" ]]; then
                    log "✓ Manifest file exists: $(basename "$file_path")"
                else
                    log "✗ Manifest file missing: $(basename "$file_path")"
                    ((FAILED_FILES++))
                fi
            fi
        done < "${backup_dir}/MANIFEST.txt"
    else
        log "! No backup manifest found for $component"
    fi
    
    # 根据组件类型检查最小预期文件
    case "$component" in
        "configs")
{% for config in backup_configs %}
{% if config.enabled %}
            if ! ls "${backup_dir}"/{{ config.name }}_*.tar.gz 1> /dev/null 2>&1; then
                log "✗ Missing {{ config.name }} configuration backup"
                ((FAILED_FILES++))
            else
                log "✓ Found {{ config.name }} configuration backup"
            fi
{% endif %}
{% endfor %}
            ;;
        "data")
{% for data in backup_data %}
{% if data.enabled %}
            if ! ls "${backup_dir}"/{{ data.name }}_*.tar.gz 1> /dev/null 2>&1; then
                log "✗ Missing {{ data.name }} data backup"
                ((FAILED_FILES++))
            else
                log "✓ Found {{ data.name }} data backup"
            fi
{% endif %}
{% endfor %}
            ;;
        "databases")
{% for db in backup_databases %}
{% if db.enabled %}
{% for database in db.databases %}
            if ! ls "${backup_dir}"/*/{{ db.name }}/{{ database }}_*.sql.gz 1> /dev/null 2>&1 && \
               ! ls "${backup_dir}"/*/{{ db.name }}/{{ database }}_*.dump.gz 1> /dev/null 2>&1; then
                log "✗ Missing {{ database }} database backup"
                ((FAILED_FILES++))
            else
                log "✓ Found {{ database }} database backup"
            fi
{% endfor %}
{% endif %}
{% endfor %}
            ;;
    esac
}

# 样本恢复测试函数
test_restore_sample() {
    local backup_dir="$1"    # 备份目录
    local component="$2"     # 组件名称
    
    log "Testing sample restore for $component from $backup_dir"
    
    # 创建测试恢复目录
    mkdir -p "$TEST_RESTORE_DIR"
    
    case "$component" in
        "configs")
            # 测试第一个找到的配置备份的恢复
            local config_backup=$(find "$backup_dir" -name "*.tar.gz" | head -n 1)
            if [[ -n "$config_backup" ]]; then
                log "Testing restore of $(basename "$config_backup")"
                
                if tar -xzf "$config_backup" -C "$TEST_RESTORE_DIR" 2>/dev/null; then
                    log "✓ Sample configuration restore test successful"
                    return 0
                else
                    log "✗ Sample configuration restore test failed"
                    ((FAILED_FILES++))
                    return 1
                fi
            else
                log "! No configuration backups found for testing"
                return 0
            fi
            ;;
        "data")
            # Test restore of first data backup found
            local data_backup=$(find "$backup_dir" -name "*.tar.gz" | head -n 1)
            if [[ -n "$data_backup" ]]; then
                log "Testing restore of $(basename "$data_backup")"
                
                if tar -xzf "$data_backup" -C "$TEST_RESTORE_DIR" 2>/dev/null; then
                    log "✓ Sample data restore test successful"
                    return 0
                else
                    log "✗ Sample data restore test failed"
                    ((FAILED_FILES++))
                    return 1
                fi
            else
                log "! No data backups found for testing"
                return 0
            fi
            ;;
        "databases")
            # 测试数据库备份的 SQL 语法
            find "$backup_dir" -name "*.sql.gz" | head -n 3 | while read -r sql_backup; do
                log "Testing SQL syntax of $(basename "$sql_backup")"
                
                if gunzip -c "$sql_backup" | head -n 100 | grep -q "CREATE\|INSERT\|UPDATE" 2>/dev/null; then
                    log "✓ SQL backup syntax appears valid: $(basename "$sql_backup")"
                else
                    log "✗ SQL backup syntax appears invalid: $(basename "$sql_backup")"
                    ((FAILED_FILES++))
                fi
            done
            ;;
    esac
}

# 备份时效性验证函数
verify_backup_age() {
    local backup_dir="$1"                    # 备份目录
    local max_age_hours="${3:-48}"           # 最大允许时间（小时），默认48小时
    
    log "Verifying backup age (max age: ${max_age_hours} hours)"
    
    local backup_timestamp=$(basename "$backup_dir")
    local backup_date
    
    # 从备份目录名称中提取日期
    if [[ "$backup_timestamp" =~ ([0-9]{8}_[0-9]{6}) ]]; then
        backup_date="${BASH_REMATCH[1]}"
        
        # 转换为时间戳
        local backup_epoch=$(date -d "${backup_date:0:8} ${backup_date:9:2}:${backup_date:11:2}:${backup_date:13:2}" +%s 2>/dev/null || echo "0")
        local current_epoch=$(date +%s)
        local age_hours=$(( (current_epoch - backup_epoch) / 3600 ))
        
        # 检查备份是否在允许的时间范围内
        if [[ "$backup_epoch" -gt 0 ]] && [[ "$age_hours" -le "$max_age_hours" ]]; then
            log "✓ Backup age is acceptable: ${age_hours} hours old"
            return 0
        else
            log "✗ Backup is too old: ${age_hours} hours old (max: ${max_age_hours})"
            ((FAILED_FILES++))
            return 1
        fi
    else
        log "! Cannot determine backup age from directory name: $backup_timestamp"
        return 0
    fi
}

# 磁盘空间验证函数
verify_disk_space() {
    log "Verifying backup disk space usage"
    
    # 获取备份目录大小和可用空间
    local backup_size=$(du -sh "$BACKUP_BASE_DIR" | cut -f1)
    local available_space=$(df -h "$BACKUP_BASE_DIR" | awk 'NR==2 {print $4}')
    
    log "Backup directory size: $backup_size"
    log "Available space: $available_space"
    
    # 检查可用空间是否少于总空间的10%
    local available_percent=$(df "$BACKUP_BASE_DIR" | awk 'NR==2 {print $4/$2*100}')
    
    if (( $(echo "$available_percent < 10" | bc -l) )); then
        log "⚠ WARNING: Low disk space - only ${available_percent}% available"
    else
        log "✓ Sufficient disk space available"
    fi
}

# 生成验证报告函数
generate_verification_report() {
    log "Generating verification report"
    
    # 生成详细的验证报告
    {
        echo "备份验证报告 (Backup Verification Report)"
        echo "=========================================="
        echo "时间戳: $(date)"
        echo "验证类型: $VERIFY_TYPE"
        echo "备份日期: $BACKUP_DATE"
        echo "主机名: $(hostname)"
        echo ""
        echo "统计摘要:"
        echo "--------"
        echo "检查文件总数: $TOTAL_FILES"
        echo "验证成功文件: $VERIFIED_FILES"
        echo "验证失败文件: $FAILED_FILES"
        echo "损坏文件数量: $CORRUPTED_FILES"
        echo ""
        
        # 根据失败文件数量确定整体状态
        if [[ "$FAILED_FILES" -eq 0 ]]; then
            echo "整体状态: ✓ 通过 (PASSED)"
        else
            echo "整体状态: ✗ 失败 (FAILED)"
        fi
        
        echo ""
        echo "详细日志: $LOG_FILE"
    } > "$VERIFICATION_REPORT"
    
    log "Verification report generated: $VERIFICATION_REPORT"
}

# 清理测试文件函数
cleanup_test_files() {
    log "Cleaning up test files"
    rm -rf "$TEST_RESTORE_DIR"  # 删除测试恢复目录
}

# 主要验证函数
# 完整性验证函数
verify_integrity() {
    log "Starting backup integrity verification"
    
    # 验证所有备份类型
    for backup_type in daily weekly monthly; do
        log "Checking $backup_type backups"
        
        # 验证每个组件
        for component in configs data databases logs; do
            local backup_dir
            if backup_dir=$(find_latest_backup "$backup_type" "$component" 2>/dev/null); then
                log "Verifying $component in $backup_dir"
                
                # 验证文件完整性
                find "$backup_dir" -type f | while read -r file; do
                    case "$file" in
                        *.tar.gz) verify_file_integrity "$file" "tar.gz" ;;
                        *.gz) verify_file_integrity "$file" "gz" ;;
                        *.bz2) verify_file_integrity "$file" "bz2" ;;
                        *.xz) verify_file_integrity "$file" "xz" ;;
                        *.sql) verify_file_integrity "$file" "sql" ;;
                    esac
                    
                    # 验证校验和
                    verify_checksum "$file"
                done
                
                # 验证备份完整性
                verify_backup_completeness "$backup_dir" "$component"
                
                # 验证备份时效性
                verify_backup_age "$backup_dir" "$component"
            fi
        done
    done
    
    # 验证磁盘空间
    verify_disk_space
}

# 恢复验证函数
verify_restore() {
    log "Starting backup restore verification"
    
    # 测试每个组件的恢复功能
    for backup_type in daily weekly monthly; do
        for component in configs data databases; do
            local backup_dir
            if backup_dir=$(find_latest_backup "$backup_type" "$component" 2>/dev/null); then
                test_restore_sample "$backup_dir" "$component"
            fi
        done
    done
}

# 计划验证函数
verify_schedule() {
    log "Starting backup schedule verification"
    
    # 检查备份是否按计划创建
    local current_hour=$(date +%H)    # 当前小时
    local current_day=$(date +%u)     # 当前星期几 (1=周一, 7=周日)
    local current_date=$(date +%d)    # 当前日期
    
    # 检查每日备份（应该存在昨天的备份）
    local yesterday=$(date -d "yesterday" +%Y%m%d)
    if find "${BACKUP_BASE_DIR}/daily" -name "*${yesterday}*" | grep -q .; then
        log "✓ Daily backup found for yesterday"
    else
        log "✗ No daily backup found for yesterday"
        ((FAILED_FILES++))
    fi
    
    # 检查每周备份（应该存在上周的备份）
    local last_week=$(date -d "7 days ago" +%Y%m%d)
    if find "${BACKUP_BASE_DIR}/weekly" -name "*${last_week}*" | grep -q .; then
        log "✓ Weekly backup found from last week"
    else
        log "! No weekly backup found from last week (may be normal)"
    fi
    
    # 检查每月备份（应该存在上月的备份）
    local last_month=$(date -d "30 days ago" +%Y%m%d)
    if find "${BACKUP_BASE_DIR}/monthly" -name "*${last_month}*" | grep -q .; then
        log "✓ Monthly backup found from last month"
    else
        log "! No monthly backup found from last month (may be normal)"
    fi
}

# 主执行函数
main() {
    log "Starting backup verification ($VERIFY_TYPE)..."
    
    # 创建必要的目录
    mkdir -p "$BACKUP_LOG_DIR"
    
    # 根据验证类型执行相应的验证
    case "$VERIFY_TYPE" in
        "integrity")
            verify_integrity      # 完整性验证
            ;;
        "restore")
            verify_restore        # 恢复验证
            ;;
        "schedule")
            verify_schedule       # 计划验证
            ;;
        "full")
            verify_integrity      # 完整验证（包含所有类型）
            verify_restore
            verify_schedule
            ;;
        *)
            error_exit "Unknown verification type: $VERIFY_TYPE. Available types: integrity, restore, schedule, full"
            ;;
    esac
    
    # 生成验证报告
    generate_verification_report
    
    # 清理临时文件
    cleanup_test_files
    
    # 最终状态检查
    if [[ "$FAILED_FILES" -eq 0 ]]; then
        log "✓ Backup verification completed successfully"
        exit 0
    else
        log "✗ Backup verification completed with $FAILED_FILES failures"
        exit 1
    fi
}

# 执行主函数
main "$@"